﻿using Sgml;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;

namespace AsianGalleriesDownloader
{
    class Program
    {
        static void Main(string[] args) {
            ServicePointManager.DefaultConnectionLimit = 10;
            try {
                string queryBaseUrl = "http://www.asian-galleries.com/index.php?s=amateur";
                for (int page = 0; ; ++page) {
                    string queryUrl = string.Format("{0}&page={1}", queryBaseUrl, page);

                    Console.WriteLine("★{0} からGallery Name, URLを取得", queryUrl);
                    List<GalleryInfo> galleryInfos = getGalleryInfo(queryUrl);
                    if (galleryInfos.Count <= 0)
                        break;

                    foreach (GalleryInfo galleryInfo in galleryInfos) {
                        string dirName = correctFileName(galleryInfo.Name);
                        string galleryDirPath = string.Format(@"D:\画像\AsianGalleries\Amateur\{0}", dirName);

                        // 出力先ディレクトリが存在する場合、ダウンロードしない
                        if (Directory.Exists(galleryDirPath))
                            continue;

                        // gallery pageから画像URLを取得
                        Console.WriteLine("●{0} から画像ページURLを取得", galleryInfo.Url);
                        List<string> imagePageUrls = getImagePageUrl(galleryInfo.Url);

                        Directory.CreateDirectory(galleryDirPath);

                        // infoファイル作成
                        createInfoFile(galleryInfo, imagePageUrls, galleryDirPath);
                        // エントリページへのショートカットファイルを作成
                        createInternetShortcut(galleryInfo.Url, galleryDirPath);

                        Console.WriteLine("●{0} の画像をダウンロード({1})", galleryInfo.Url, imagePageUrls.Count);
                        downloadImages(imagePageUrls, galleryDirPath);
                    }
                }
            }
            catch (Exception exp) {
                Console.WriteLine(exp.ToString());
            }
            Console.WriteLine("終了");
            Console.ReadKey();
        }

        static string correctFileName(string src) {
            src.Trim();
            char[] invalidChar = Path.GetInvalidFileNameChars();
            foreach (char c in invalidChar) {
                src = src.Replace(c, '_');
            }
            return src;
        }

        static Encoding _enc = Encoding.GetEncoding("utf-8");

        static void createInfoFile(GalleryInfo galleryInfo, List<string> imageUrls, string outputDir) {
            string infoFilePath = Path.Combine(outputDir, "_info.txt");
            using (StreamWriter infoWriter = new StreamWriter(infoFilePath, true, _enc)) {
                infoWriter.WriteLine(galleryInfo.Url);
                infoWriter.WriteLine(galleryInfo.Name);
                infoWriter.WriteLine("");
                foreach (string imageUrl in imageUrls) {
                    infoWriter.WriteLine(imageUrl);
                }
            }
        }

        static void createInternetShortcut(string targetUrl, string outputDir) {
            string linkFilePath = Path.Combine(outputDir, "_galleryPage.url");
            using (StreamWriter sw = new StreamWriter(linkFilePath, false, _enc)) {
                sw.WriteLine("[InternetShortcut]");
                sw.WriteLine("URL=" + targetUrl);
            }
        }

        class FailedInfo
        {
            public string Url;
            public string Message;
        }

        static void downloadImages(List<string> imagePageUrls, string outDir) {
            List<FailedInfo> failedInfos = new List<FailedInfo>();

            Parallel.ForEach(imagePageUrls, new ParallelOptions() { MaxDegreeOfParallelism = 3 }, (imagePageUrl, state, index) => {
                try {
                    downloadImage(getImageUrlFromImagePage(imagePageUrl), (int)index, outDir);
                    Console.Write("o");
                }
                catch (Exception exp) {
                    Console.Write("x");
                    FailedInfo failedInfo = new FailedInfo() { Url = imagePageUrl };

                    WebException we = exp as WebException;
                    if (null != we && we.Status == WebExceptionStatus.ProtocolError) {
                        HttpStatusCode statusCode = ((HttpWebResponse)we.Response).StatusCode;
                        // とりあえず 404 以外の場合は処理中止
                        if (HttpStatusCode.NotFound == statusCode)
                            failedInfo.Message = statusCode.ToString();
                    }
                    else if (exp is IOException) {
                        failedInfo.Message = exp.Message;
                    }

                    if (null == failedInfo.Message)
                        throw exp;

                    lock (failedInfos) {
                        failedInfos.Add(failedInfo);
                    }
                }
            });
            Console.Write("\n");

            // ダウンロードに失敗したファイルを failed～.txt に出力
            if (0 < failedInfos.Count) {
                DateTime now = DateTime.Now;
                string failedFilePath = Path.Combine(outDir, string.Format("_failed_{0}.txt", now.ToString("yyMMdd_HHmmss")));
                using (StreamWriter writer = new StreamWriter(failedFilePath, true, _enc)) {
                    foreach (FailedInfo failedInfo in failedInfos) {
                        writer.WriteLine(failedInfo.Url + "\t" + failedInfo.Message);
                    }
                }
            }
        }

        class GalleryInfo
        {
            public string Name;
            public string Url;
        }

        static List<GalleryInfo> getGalleryInfo(string queryUrl) {
            WebResponse res = getWebResponse(queryUrl);

            string src;

            using (Stream st = res.GetResponseStream())
            using (StreamReader sr = new StreamReader(st, _enc)) {
                src = sr.ReadToEnd();
            }

            XmlDocument xmlDoc = new XmlDocument();
            using (SgmlReader sgmlReader = new SgmlReader()) {
                sgmlReader.DocType = "HTML";
                sgmlReader.CaseFolding = CaseFolding.ToLower;
                sgmlReader.InputStream = new StringReader(src);
                xmlDoc.Load(sgmlReader);
            }

            //xmlDoc.Save("test.xml");

            List<GalleryInfo> ret = new List<GalleryInfo>();

            // Gallery Name, URL
            // <a href="http://www.asian-galleries.com/gallery/3991592/Indian-Lolita">Indian Lolita</a>
            XmlNodeList nodes = xmlDoc.SelectNodes("//tr[@class='row']/td/a");
            foreach (XmlNode node in nodes) {
                GalleryInfo info = new GalleryInfo();
                ret.Add(info);

                info.Url = node.Attributes["href"].Value;
                info.Name = node.InnerText;
            }

            return ret;
        }

        static List<string> getImagePageUrl(string url) {
            WebResponse res = getWebResponse(url);

            string src;

            using (Stream st = res.GetResponseStream())
            using (StreamReader sr = new StreamReader(st, _enc)) {
                src = sr.ReadToEnd();
            }

            XmlDocument xmlDoc = new XmlDocument();
            using (SgmlReader sgmlReader = new SgmlReader()) {
                sgmlReader.DocType = "HTML";
                sgmlReader.CaseFolding = CaseFolding.ToLower;
                sgmlReader.InputStream = new StringReader(src);
                xmlDoc.Load(sgmlReader);
            }

            //xmlDoc.Save("test.xml");

            List<string> ret = new List<string>();

            // 画像ページ
            // xpath: /html/body/div/div[3]/table//a
            // <a href="http://www.asian-galleries.com/watch/1792145116/Lolita-Cheng-Set-251">
            XmlNodeList nodes = xmlDoc.SelectNodes("/html/body/div/div[3]/table//a");
            foreach (XmlNode node in nodes) {
                string imagePageUrl = node.Attributes["href"].Value;
                ret.Add(imagePageUrl);
            }

            return ret;
        }

        static string getImageUrlFromImagePage(string url) {
            string ret = null;
            do {
                WebResponse res = getWebResponse(url);

                string src;

                using (Stream st = res.GetResponseStream())
                using (StreamReader sr = new StreamReader(st, _enc)) {
                    src = sr.ReadToEnd();
                }

                XmlDocument xmlDoc = new XmlDocument();
                using (SgmlReader sgmlReader = new SgmlReader()) {
                    sgmlReader.DocType = "HTML";
                    sgmlReader.CaseFolding = CaseFolding.ToLower;
                    sgmlReader.InputStream = new StringReader(src);
                    xmlDoc.Load(sgmlReader);
                }

                //xmlDoc.Save("test.xml");

                // 画像
                // xpath: //div[@id='flow']/img
                XmlNode node = xmlDoc.SelectSingleNode("//div[@id='flow']/img");
                ret = node.Attributes["src"].Value;

                // src="http://www.asian-galleries.com/view.php?pic="の場合がたまにあって
                // 何度かリトライすると取得できたのでチェック
                Regex reg = new Regex(@"view.php\?pic=(?<url>.*?)$", RegexOptions.IgnoreCase);
                Match mat = reg.Match(ret);
                if (!mat.Success || String.IsNullOrEmpty(mat.Groups["url"].Value)) {
                    Console.WriteLine("\n" + "画像URLの取得をリトライ");
                    ret = null;
                    Thread.Sleep(TimeSpan.FromSeconds(10));
                }
            } while(null == ret);
            return ret;
        }


        static void downloadImage(string url, int index, string dir) {
            // 保存ファイルパスを作成
            string filePath;
            // "http://www.asian-galleries.com/view.php?pic=http%3A%2F%2Ffap.to%2Fimages%2F50%2F1073820611%2Fasian-porn%2FIndian-Lolita-19.jpg"
            // → "http://www.asian-galleries.com/view.php?pic=http://fap.to/images/50/1073820611/asian-porn/Indian-Lolita-19.jpg"
            string decodedUrl = Uri.UnescapeDataString(url);
            decodedUrl = Uri.UnescapeDataString(decodedUrl);
            // ファイル名に使えない文字を含んでいると Path.GetFileName で例外が出るので使わない
            string fileName = decodedUrl.Substring(decodedUrl.LastIndexOf('/') + 1);
            filePath = Path.Combine(dir, correctFileName(fileName));

            // ファイルが存在している場合、ダウンロードしない
            if (File.Exists(filePath))
                return;

            // WebRequestの作成
            //サーバーからの応答を受信するためのWebResponseを取得
            WebResponse webres = getWebResponse(url);

            // メモリに格納
            MemoryStream ms = new MemoryStream();

            //応答データを受信するためのStreamを取得
            using (Stream rs = webres.GetResponseStream()) {
                rs.ReadTimeout = 1000 * 60 * 5;
                byte[] buffer = new byte[65535];
                while (true) {
                    int rb = rs.Read(buffer, 0, buffer.Length);
                    if (rb > 0) {
                        ms.Write(buffer, 0, rb);
                    }
                    else {
                        break;
                    }
                }
            }

            // ファイルに保存
            FileStream fs = new FileStream(filePath, FileMode.Create);
            fs.Write(ms.GetBuffer(), 0, (int)ms.Length);
            fs.Close();
        }

        static WebResponse getWebResponse(string url) {
            WebResponse res = null;
            int retry = 0;
            do {
                try {
                    WebRequest req = WebRequest.Create(url);
                    req.Timeout = 1000 * 60 * 5;
                    res = req.GetResponse();
                }
                catch (WebException we) {
                    // とりあえず404以外の場合、6回繰り返す。
                    if (retry++ >= 6 ||
                        (we.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)we.Response).StatusCode == HttpStatusCode.NotFound)) {
                        throw we;
                    }
                    Console.WriteLine("\n" + DateTime.Now.ToString() + "\n" + we.ToString());
                    Thread.Sleep(TimeSpan.FromSeconds(10));
                    continue;
                }
            } while (null == res);
            return res;
        }

    }

}
